import requests
from lxml import etree
import pymongo

client = pymongo.MongoClient()
db = client['toscrape']
collection = db['toscrape']

page = 1
data = []
while True:
    url = f'http://quotes.toscrape.com/page/{page}/'
    res = requests.get(url)
    tree = etree.HTML(res.text)
    items = tree.xpath('//div[@class="quote"]')
    for item in items:
        content = item.xpath("./span[@class='text']/text()")[0]
        autor = item.xpath(".//small[@class='author']/text()")[0]
        tags = item.xpath(".//a[@class='tag']/text()")
        data.append({
            'content': content,
            'autor': autor,
            'tags': tags
        })
    print(f'第{page}页结束')

    next_btn = tree.xpath('//li[@class="next"]')
    if next_btn:
        page += 1
    else:
        break
collection.insert_many(data)

client.close()
